Creating Anomaly Detection System for Industrial Process with Deep Learning
This is an R Markdown Notebook
Your project - Detect Anomaly in Industrial Process with Deep Learning
Deep Learning… In this lecture I would like to esplore with you the DeepLearning we can do with H2O package in R…
Data Exploration
names(DF_NORMAL)
[1] "DateCheck" "Inf1" "Inf2" "Inf3" "Inf4" "Centr1" "Centr2" "Out1" "Out2"
[10] "Out3" "Out4" about the dataset
summary(NORM_2016)
Inf1 Inf2 Inf3 Inf4 Centr1 Centr2 Out1 Out2
Min. :1060 Min. :1004 Min. :1698 Min. :1698 Min. :2695 Min. :3300 Min. :1360 Min. :1593
1st Qu.:1110 1st Qu.:1360 1st Qu.:1945 1st Qu.:1998 1st Qu.:2802 1st Qu.:3380 1st Qu.:1660 1st Qu.:2012
Median :1130 Median :1427 Median :1998 Median :2043 Median :2841 Median :3430 Median :1708 Median :2068
Mean :1131 Mean :1423 Mean :1993 Mean :2042 Mean :2843 Mean :3429 Mean :1705 Mean :2066
3rd Qu.:1150 3rd Qu.:1489 3rd Qu.:2041 3rd Qu.:2091 3rd Qu.:2886 3rd Qu.:3470 3rd Qu.:1757 3rd Qu.:2121
Max. :1200 Max. :1684 Max. :2202 Max. :2295 Max. :3057 Max. :3660 Max. :2026 Max. :2316
Out3 Out4
Min. :46.51 Min. :66.51
1st Qu.:64.79 1st Qu.:80.00
Median :68.84 Median :82.59
Mean :69.01 Mean :82.60
3rd Qu.:73.33 3rd Qu.:85.37
Max. :86.91 Max. :94.74 This dataset was selected .. after .. the period of deep process maintenance
NORM_2016 <- DF_NORMAL %>% select(2:11) %>% as.matrix()
plot_ly(z = NORM_2016, type = "surface")
TEST_2015 <- DF_TEST %>% select(2:11) %>% as.matrix()
plot_ly(z = TEST_2015, type = "surface")
TEST_2017 <- DF_ANOMALY %>% select(2:11) %>% as.matrix()
plot_ly(z = TEST_2017, type = "surface")
Train Deep Learning Model DRAFT
NORM_2016_sc <- DF_NORMAL %>% select(2:11) %>% scale() %>% as.matrix() normality_model <- h2o.deeplearning(x = names(train),
training_frame = train,
activation = "Tanh",
autoencoder = TRUE,
hidden = c(200,100,200),
sparse = TRUE,
l1 = 1e-4,
epochs = 100)
|
| | 0%
|
|=== | 3%
|
|======= | 6%
|
|========== | 9%
|
|============= | 12%
|
|================= | 15%
|
|==================== | 18%
|
|======================== | 21%
|
|=========================== | 24%
|
|============================== | 27%
|
|================================== | 30%
|
|================================================================================================================| 100%normality_model_sc <- h2o.deeplearning(x = names(train_sc),
training_frame = train_sc,
activation = "Tanh",
autoencoder = TRUE,
hidden = c(200,100,200),
sparse = TRUE,
l1 = 1e-4,
epochs = 100)
|
| | 0%
|
|=== | 3%
|
|======= | 6%
|
|========== | 9%
|
|============== | 12%
|
|================= | 15%
|
|===================== | 18%
|
|======================== | 22%
|
|============================ | 25%
|
|=============================== | 28%
|
|=================================== | 31%
|
|====================================== | 34%
|
|========================================= | 37%
|
|============================================= | 40%
|
|================================================ | 43%
|
|==================================================== | 46%
|
|======================================================= | 49%
|
|=========================================================== | 52%
|
|============================================================== | 55%
|
|================================================================== | 59%
|
|===================================================================== | 62%
|
|================================================================================================================| 100%reconstruct dataset
Check MSE
shutdown JVM
h2o.shutdown(prompt = F)
Error in h2o.shutdown(prompt = F) : There is no H2O instance running.LS0tDQp0aXRsZTogIkNyZWF0aW5nIEFub21hbHkgRGV0ZWN0aW9uIFN5c3RlbSBmb3IgSW5kdXN0cmlhbCBQcm9jZXNzIHdpdGggRGVlcCBMZWFybmluZyINCm91dHB1dDoNCiAgaHRtbF9kb2N1bWVudDogZGVmYXVsdA0KICBodG1sX25vdGVib29rOiBkZWZhdWx0DQogIHBkZl9kb2N1bWVudDogZGVmYXVsdA0KLS0tDQoNClRoaXMgaXMgYW4gW1IgTWFya2Rvd25dKGh0dHA6Ly9ybWFya2Rvd24ucnN0dWRpby5jb20pIE5vdGVib29rDQoNCg0KIyMjIFlvdXIgcHJvamVjdCAtIERldGVjdCBBbm9tYWx5IGluIEluZHVzdHJpYWwgUHJvY2VzcyB3aXRoIERlZXAgTGVhcm5pbmcNCg0KRGVlcCBMZWFybmluZy4uLiBJbiB0aGlzIGxlY3R1cmUgSSB3b3VsZCBsaWtlIHRvIGVzcGxvcmUgd2l0aCB5b3UgdGhlIERlZXBMZWFybmluZyB3ZSBjYW4gZG8gd2l0aCBIMk8gcGFja2FnZSBpbiBSLi4uDQoNCiMjIyMgRGF0YSBFeHBsb3JhdGlvbg0KDQpgYGB7cn0NCiMgbGlicmFyaWVzDQpsaWJyYXJ5KHRpZHl2ZXJzZSkNCmxpYnJhcnkocGxvdGx5KQ0KDQojIGRhdGEgcmVhZGluZw0KREZfTk9STUFMIDwtIHJlYWRfcmRzKCJEQVRBLW5vcm1hbC5yZHMiKQ0KREZfVEVTVCA8LSByZWFkX3JkcygiREFUQS10ZXN0LnJkcyIpDQpERl9BTk9NQUxZIDwtIHJlYWRfcmRzKCJEQVRBLWFub21hbHkucmRzIikNCg0KIyBuYW1lcyBvZiB0aGUgZGF0YXNldCBjb2x1bW5zDQpuYW1lcyhERl9OT1JNQUwpDQpgYGANCg0KYWJvdXQgdGhlIGRhdGFzZXQNCg0KYGBge3J9DQpOT1JNXzIwMTYgPC0gREZfTk9STUFMICU+JSBzZWxlY3QoMjoxMSkgJT4lIGFzLm1hdHJpeCgpIA0Kc3VtbWFyeShOT1JNXzIwMTYpDQoNCmBgYA0KVGhpcyBkYXRhc2V0IHdhcyBzZWxlY3RlZCAuLiBhZnRlciAuLiB0aGUgcGVyaW9kIG9mIGRlZXAgcHJvY2VzcyBtYWludGVuYW5jZQ0KDQpgYGB7cn0NCnBsb3RfbHkoeiA9IE5PUk1fMjAxNiwgdHlwZSA9ICJzdXJmYWNlIikNCmBgYA0KDQoNCg0KDQpgYGB7cn0NCg0KDQoNCg0KVEVTVF8yMDE1IDwtIERGX1RFU1QgJT4lIHNlbGVjdCgyOjExKSAlPiUgYXMubWF0cml4KCkNCnBsb3RfbHkoeiA9IFRFU1RfMjAxNSwgdHlwZSA9ICJzdXJmYWNlIikNCg0KVEVTVF8yMDE3IDwtIERGX0FOT01BTFkgJT4lIHNlbGVjdCgyOjExKSAlPiUgYXMubWF0cml4KCkNCnBsb3RfbHkoeiA9IFRFU1RfMjAxNywgdHlwZSA9ICJzdXJmYWNlIikNCg0KDQpgYGANCg0KYGBge3J9DQpOT1JNXzIwMTYgPC0gREZfTk9STUFMICU+JSBzZWxlY3QoMjoxMSkgJT4lIGFzLm1hdHJpeCgpDQoNCg0KDQpwbG90X2x5KHogPSBOT1JNXzIwMTYsIHR5cGUgPSAic3VyZmFjZSIpDQoNClRFU1RfMjAxNSA8LSBERl9URVNUICU+JSBzZWxlY3QoMjoxMSkgJT4lIGFzLm1hdHJpeCgpDQpwbG90X2x5KHogPSBURVNUXzIwMTUsIHR5cGUgPSAic3VyZmFjZSIpDQoNClRFU1RfMjAxNyA8LSBERl9BTk9NQUxZICU+JSBzZWxlY3QoMjoxMSkgJT4lIGFzLm1hdHJpeCgpDQpwbG90X2x5KHogPSBURVNUXzIwMTcsIHR5cGUgPSAic3VyZmFjZSIpDQoNCg0KYGBgDQoNCg0KDQoNCg0KDQoNCg0KIyMjIyBUcmFpbiBEZWVwIExlYXJuaW5nIE1vZGVsIERSQUZUDQoNCmBgYHtyfQ0KTk9STV8yMDE2X3NjIDwtIERGX05PUk1BTCAlPiUgc2VsZWN0KDI6MTEpICU+JSBzY2FsZSgpICU+JSBhcy5tYXRyaXgoKSANCmBgYA0KDQoNCg0KYGBge3J9DQpsaWJyYXJ5KGgybykNCmgyby5pbml0KG50aHJlYWRzID0gMikNCnRyYWluX3NjIDwtIGFzLmgybyh4ID0gTk9STV8yMDE2X3NjLCBkZXN0aW5hdGlvbl9mcmFtZSA9ICJ0cmFpbnNjIikNCnRyYWluIDwtIGFzLmgybyh4ID0gTk9STV8yMDE2LCBkZXN0aW5hdGlvbl9mcmFtZSA9ICJ0cmFpbiIpDQp0ZXN0IDwtIGFzLmgybyh4ID0gVEVTVF8yMDE1LCBkZXN0aW5hdGlvbl9mcmFtZSA9ICJ0ZXN0IikNCmFub21hbHkgPC0gYXMuaDJvKHggPSBURVNUXzIwMTcsIGRlc3RpbmF0aW9uX2ZyYW1lID0gImFub21hbHkiKQ0KDQpub3JtYWxpdHlfbW9kZWwgPC0gaDJvLmRlZXBsZWFybmluZyh4ID0gbmFtZXModHJhaW4pLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0cmFpbmluZ19mcmFtZSA9IHRyYWluLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhY3RpdmF0aW9uID0gIlRhbmgiLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhdXRvZW5jb2RlciA9IFRSVUUsIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGhpZGRlbiA9IGMoMjAwLDEwMCwyMDApLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzcGFyc2UgPSBUUlVFLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGwxID0gMWUtNCwgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZXBvY2hzID0gMTAwKQ0KDQpub3JtYWxpdHlfbW9kZWxfc2MgPC0gaDJvLmRlZXBsZWFybmluZyh4ID0gbmFtZXModHJhaW5fc2MpLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0cmFpbmluZ19mcmFtZSA9IHRyYWluX3NjLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhY3RpdmF0aW9uID0gIlRhbmgiLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhdXRvZW5jb2RlciA9IFRSVUUsIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGhpZGRlbiA9IGMoMjAwLDEwMCwyMDApLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzcGFyc2UgPSBUUlVFLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGwxID0gMWUtNCwgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZXBvY2hzID0gMTAwKQ0KDQpgYGANCg0KIyMjIyByZWNvbnN0cnVjdCBkYXRhc2V0DQoNCmBgYHtyfQ0KIyByZWNyZWF0ZSANCnRlc3RfcmVjb24gPC0gaDJvLnByZWRpY3Qobm9ybWFsaXR5X21vZGVsLCB0cmFpbikgJT4lIGFzLm1hdHJpeCgpDQpwbG90X2x5KHogPSB0ZXN0X3JlY29uLCB0eXBlID0gInN1cmZhY2UiKQ0KYGBgDQoNCiMjIyMgQ2hlY2sgTVNFDQoNCg0KYGBge3J9DQoNCm1zZSA8LSBoMm8uYW5vbWFseShub3JtYWxpdHlfbW9kZWwsIHRyYWluKSAlPiUgYXMuZGF0YS5mcmFtZSgpDQptc2Vfc2MgPC0gaDJvLmFub21hbHkobm9ybWFsaXR5X21vZGVsX3NjLCB0cmFpbl9zYykgJT4lIGFzLmRhdGEuZnJhbWUoKQ0KbXNlX3Rlc3QgPC0gaDJvLmFub21hbHkobm9ybWFsaXR5X21vZGVsLCB0ZXN0KSAlPiUgYXMuZGF0YS5mcmFtZSgpDQptc2VfYW5vbSA8LSBoMm8uYW5vbWFseShub3JtYWxpdHlfbW9kZWwsIGFub21hbHkpICU+JSBhcy5kYXRhLmZyYW1lKCkNCg0KcGxvdC50cyhtc2UpDQpwbG90LnRzKG1zZV9zYykNCnBsb3QudHMobXNlX3Rlc3QpDQpwbG90LnRzKG1zZV9hbm9tKQ0KDQoNCmBgYA0KDQojIHNodXRkb3duIEpWTQ0KYGBge3J9DQpoMm8uc2h1dGRvd24ocHJvbXB0ID0gRikNCmBgYA0KDQo=